package com.example.demo.admin.gecco.crawler;/**
 * Created by kete on 2017-11-07.
 */

import com.example.demo.admin.business.IpProxyBusiness;
import com.example.demo.admin.gecco.entity.XichiIpProxyInfo;
import com.example.demo.common.persistence.dao.IpProxyMapper;
import com.example.demo.common.persistence.model.IpProxy;
import com.geccocrawler.gecco.annotation.PipelineName;
import com.geccocrawler.gecco.pipeline.Pipeline;
import com.geccocrawler.gecco.request.HttpGetRequest;
import com.geccocrawler.gecco.scheduler.SchedulerContext;
import org.jsoup.Jsoup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.List;

/**
 * 作者：kete
 * 创建时间：2017-11-07 16:07
 * 功能描述：
 * 版本：
 */
@PipelineName("xichiSaveInfoPipeline")
@Service
public class XichiSaveInfoPipeline implements Pipeline<XichiPage> {

    private static Logger logger = LoggerFactory.getLogger(XichiSaveInfoPipeline.class);

    @Autowired
    IpProxyMapper ipProxyMapper;

    @Autowired
    IpProxyBusiness ipProxyBusiness;

    @Override
    public void process(XichiPage indexPage) {

        List<XichiIpProxyInfo> list = indexPage.getList();
        for(XichiIpProxyInfo proxyInfo : list){

            if(!ipProxyBusiness.checkProxy(proxyInfo)) continue;

            ipProxyBusiness.insertIpProxy(proxyInfo);
        }

        String nextUrl = indexPage.getNextUrl();

        Integer pageIndex = indexPage.getPageIndex();

        if(pageIndex<4){
//            pageIndex = pageIndex+1;
//
//            nextUrl = nextUrl.substring(0,nextUrl.length()-2)+pageIndex;

            logger.info("nextUrl:"+nextUrl);

            HttpGetRequest sub = (HttpGetRequest)indexPage.getRequest().subRequest(nextUrl);

            SchedulerContext.into(sub);

        }




    }



}
